# -*- coding: utf-8 -*-
import scrapy
import sys
import codecs
sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
from douban.items import DoubanItem

class DoubanSpiderSpider(scrapy.Spider):
    # 爬出明
    name = 'douban_spider'
    # 允许的域名
    allowed_domains = ['movie.douban.com']
    # 入口url，扔到调度器里面
    start_urls = ['https://movie.douban.com/top250']

    # 默认解析方法
    def parse(self, response):
        movie_list = response.xpath("//div[@class='article']/ol[@class='grid_view']/li")
        for item in movie_list:
            douban_item = DoubanItem()
            douban_item['serial_number'] = item.xpath(".//div[@class='item']//em/text()").extract_first()
            douban_item['movie_name'] = item.xpath(".//div[@class='info']/div[@class='hd']/a/span[1]/text()").extract_first()
            # 电影详情
            contents = item.xpath(".//div[@class='info']//div[@class='bd']/p[1]/text()").extract()
            for content in contents:
                douban_item['introduce'] = "".join(content.split())
            douban_item['star'] = item.xpath(".//span[@class='rating_num']/text()").extract_first()
            douban_item['evaluate'] = item.xpath(".//div[@class='star']//span[4]/text()").extract_first()
            douban_item['descript'] = item.xpath(".//p[@class='quote']/span/text()").extract_first()
            # 需要将数据yield到pipelines中去
            yield douban_item

        # next page
        next_links = response.xpath("//span[@class='next']/link/@href").extract()
        if next_links:
            next_link = next_links[0]
            yield scrapy.Request("https://movie.douban.com/top250" + next_link, callback=self.parse)
